# Gestione warning e logging
import warnings
import logging
from typing import Dict, List, Tuple, Optional, Any, NamedTuple, Union
from dataclasses import dataclass, field
from pathlib import Path
import sys
import os
from functools import lru_cache
from concurrent.futures import ThreadPoolExecutor
import re
from datetime import datetime

# Disabilita warnings
warnings.filterwarnings('ignore')

# Setup logging base
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

# Librerie di base per l'analisi dati
import pandas as pd
import numpy as np
from pandas.api.types import is_numeric_dtype

# Librerie per visualizzazione
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import matplotlib.pyplot as plt
import seaborn as sns

@dataclass
class PlotConfig:
    COLOR_PALETTE: Dict[str, str] = None
    PLOT_STYLE: Dict[str, Any] = None

    def __post_init__(self):
        self.COLOR_PALETTE = {
            'primary': '#2c3e50',
            'secondary': '#34495e',
            'success': '#27ae60',
            'warning': '#f39c12',
            'danger': '#c0392b',
            'info': '#3498db'
        }

        self.PLOT_STYLE = {
            'template': 'plotly_white',
            'font_family': 'Arial, sans-serif',
            'title_font_size': 20,
            'title_x': 0.5,
            'showlegend': True
        }

class DataFormatter:

    @staticmethod
    @lru_cache(maxsize=1000)
    def format_number(num: Union[int, float]) -> str:
        """Formatta numeri con separatori di migliaia"""
        return f"{num:,.0f}"

    @staticmethod
    @lru_cache(maxsize=1000)
    def format_percentage(num: Union[int, float]) -> str:
        """Formatta percentuali con 1 decimale"""
        return f"{num:.1f}%"

    @staticmethod
    @lru_cache(maxsize=1000)
    def format_currency(num: Union[int, float]) -> str:
        """Formatta valori monetari"""
        return f"${num:,.2f}"

class DataLoader:

    def __init__(self):
        self.plot_config = PlotConfig()

    def _is_colab_environment(self) -> bool:
        try:
            import google.colab
            return True
        except ImportError:
            return False

    def _setup_visualization_settings(self):

        # Impostazioni pandas
        pd.set_option('display.max_columns', None)
        pd.set_option('display.max_rows', 100)
        pd.set_option('display.float_format', lambda x: '%.3f' % x)

        # Impostazioni matplotlib/seaborn
        plt.style.use('default')
        sns.set_theme(style='whitegrid')

    def load_data(self) -> Tuple[Optional[pd.DataFrame], Optional[pd.DataFrame]]:
        try:
            # Setup visualizzazione
            self._setup_visualization_settings()

            # Determina l' ambiente e carica i dati
            if self._is_colab_environment():
                logger.info("Ambiente rilevato: Google Colab")
                from google.colab import files
                logger.info("Per favore, carica i file 'googleplaystore.csv' e 'googleplaystore_user_reviews.csv'")
                uploaded = files.upload()

            # Verifica presenza file
            required_files = ['googleplaystore.csv', 'googleplaystore_user_reviews.csv']
            for file in required_files:
                if not os.path.exists(file):
                    raise FileNotFoundError(f"File {file} non trovato nella directory corrente")

            apps_df = pd.read_csv('googleplaystore.csv')
            reviews_df = pd.read_csv('googleplaystore_user_reviews.csv')

            logger.info(f"Dataset caricati con successo!")
            logger.info(f"Dimensioni apps_df: {apps_df.shape}")
            logger.info(f"Dimensioni reviews_df: {reviews_df.shape}")

            return apps_df, reviews_df

        except Exception as e:
            logger.error(f"Errore nel caricamento dei dati: {str(e)}")
            return None, None

# Inizializzazione del loader e caricamento dei dati
data_loader = DataLoader()
apps_df, reviews_df = data_loader.load_data()

# Verifica che i dati siano stati caricati correttamente
if apps_df is None or reviews_df is None:
    logger.error("Errore nel caricamento dei dataset. Verifica la presenza dei file o ricaricali.")
    sys.exit(1)

Saving googleplaystore_user_reviews.csv to googleplaystore_user_reviews.csv
Saving googleplaystore.csv to googleplaystore.csv

from functools import partial


logger = logging.getLogger(__name__)

@dataclass
class DatasetMetrics:
    rows: int
    columns: int
    missing_data: Dict[str, float]
    duplicates: int
    duplicate_percentage: float
    dtypes: Dict[str, str]
    unique_counts: Dict[str, int]

class DataValidator:

    def __init__(self, max_workers: int = 4):
        self.max_workers = max_workers

    @staticmethod
    def _calculate_column_metrics(df: pd.DataFrame, column: str) -> Dict[str, Any]:
        return {
            'tipo': str(df[column].dtype),
            'non_nulli': df[column].count(),
            'nulli': df[column].isnull().sum(),
            'perc_nulli': (df[column].isnull().sum() / len(df) * 100).round(2),
            'unique_values': df[column].nunique()
        }

    def validate_dataset(self, df: pd.DataFrame, dataset_name: str) -> DatasetMetrics:
        logger.info(f"\nValidazione dataset: {dataset_name}")
        logger.info("-" * 50)

        # Calcolo metriche base
        rows, cols = df.shape
        logger.info(f"Dimensioni: {rows:,} righe, {cols} colonne")

        # Calcolo metriche per colonna in parallelo
        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            column_metrics = dict(zip(
                df.columns,
                executor.map(partial(self._calculate_column_metrics, df), df.columns)
            ))

        # Calcolo duplicati
        duplicates = df.duplicated().sum()
        duplicate_percentage = (duplicates/len(df)*100).round(2)

        logger.info(f"\nDuplicati trovati: {duplicates:,} ({duplicate_percentage}%)")

        return DatasetMetrics(
            rows=rows,
            columns=cols,
            missing_data={col: metrics['perc_nulli']
                         for col, metrics in column_metrics.items()},
            duplicates=duplicates,
            duplicate_percentage=duplicate_percentage,
            dtypes={col: metrics['tipo']
                   for col, metrics in column_metrics.items()},
            unique_counts={col: metrics['unique_values']
                         for col, metrics in column_metrics.items()}
        )

class DataConsistencyChecker:

    @staticmethod
    def check_consistency(apps_df: pd.DataFrame,
                         reviews_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
        logger.info("\nVerifica consistenza tra dataset")
        logger.info("-" * 50)

        # Verifica app presenti in entrambi i dataset
        apps_in_store = set(apps_df['App'].unique())
        apps_in_reviews = set(reviews_df['App'].unique())

        common_apps = apps_in_store.intersection(apps_in_reviews)
        missing_apps = apps_in_reviews - apps_in_store

        logger.info(f"App nel Play Store: {len(apps_in_store):,}")
        logger.info(f"App con recensioni: {len(apps_in_reviews):,}")
        logger.info(f"App in comune: {len(common_apps):,}")

        if missing_apps:
            logger.warning(
                f"\nAttenzione: {len(missing_apps):,} app hanno recensioni "
                "ma non sono nel dataset principale"
            )

        # Verifica integrità
        logger.info("\nVerifica integrità dei dati:")
        logger.info(f"- App univoche nel Play Store: {apps_df['App'].nunique():,}")
        logger.info(f"- Record totali recensioni: {len(reviews_df):,}")

        return apps_df, reviews_df

class InitialAnalyzer:

    @staticmethod
    def analyze_dataset(df: pd.DataFrame, dataset_name: str) -> None:
        logger.info(f"\nAnalisi iniziale: {dataset_name}")
        logger.info("-" * 50)

        # Analisi colonne numeriche
        numeric_cols = df.select_dtypes(include=[np.number]).columns
        if len(numeric_cols) > 0:
            logger.info("\nStatistiche colonne numeriche:")
            logger.info(df[numeric_cols].describe().round(2))

        # Analisi colonne categoriche
        categorical_cols = df.select_dtypes(include=['object']).columns
        if len(categorical_cols) > 0:
            logger.info("\nStatistiche colonne categoriche:")
            for col in categorical_cols:
                unique_vals = df[col].nunique()
                logger.info(f"\n{col}:")
                logger.info(f"- Valori unici: {unique_vals:,}")
                if unique_vals < 10:
                    dist = df[col].value_counts(normalize=True).head()
                    logger.info(f"Distribuzione:\n{dist.round(3)}")

def validate_and_analyze_data(apps_df: pd.DataFrame,
                            reviews_df: pd.DataFrame,
                            validator: Optional[DataValidator] = None,
                            consistency_checker: Optional[DataConsistencyChecker] = None,
                            initial_analyzer: Optional[InitialAnalyzer] = None) -> Tuple[pd.DataFrame, pd.DataFrame]:
    logger.info("=== INIZIO VALIDAZIONE E ANALISI DATI ===")

    # Inizializzazione componenti se non forniti
    validator = validator or DataValidator()
    consistency_checker = consistency_checker or DataConsistencyChecker()
    initial_analyzer = initial_analyzer or InitialAnalyzer()

    try:
        # Validazione dataset
        apps_metrics = validator.validate_dataset(apps_df, "Google Play Store Apps")
        reviews_metrics = validator.validate_dataset(reviews_df, "App Reviews")

        # Verifica della consistenza
        apps_df, reviews_df = consistency_checker.check_consistency(apps_df, reviews_df)

        # Analisi iniziale
        initial_analyzer.analyze_dataset(apps_df, "Google Play Store Apps")
        initial_analyzer.analyze_dataset(reviews_df, "App Reviews")

        return apps_df, reviews_df

    except Exception as e:
        logger.error(f"Errore durante la validazione e analisi: {str(e)}")
        raise

# Esecuzione della validazione e analisi
apps_df, reviews_df = validate_and_analyze_data(apps_df, reviews_df)

WARNING:__main__:
Attenzione: 54 app hanno recensioni ma non sono nel dataset principale

logger = logging.getLogger(__name__)

@dataclass
class CleaningReport:
    original_rows: int
    cleaned_rows: int
    removed_rows: int
    missing_before: Dict[str, int]
    missing_after: Dict[str, int]
    cleaning_steps: list[str]

class DataCleaner:

    @staticmethod
    def clean_size(size: str) -> Optional[float]:
        """Converte la dimensione dell'app in MB"""
        if pd.isna(size) or size == 'Varies with device':
            return np.nan
        try:
            size_str = str(size).strip().upper()
            multiplier = 1/1024 if 'K' in size_str else 1
            return float(re.sub(r'[^0-9.]', '', size_str)) * multiplier
        except (ValueError, AttributeError):
            return np.nan

    @staticmethod
    def clean_price(price: str) -> float:
        """Converte il prezzo in valore numerico"""
        if pd.isna(price) or price in ['Free', '0', 'Everyone']:
            return 0.0
        try:
            return float(re.sub(r'[^0-9.]', '', str(price)))
        except (ValueError, AttributeError):
            return 0.0

    @staticmethod
    def clean_installs(installs: str) -> int:
        """Converte il numero di installazioni in valore numerico"""
        if pd.isna(installs):
            return 0
        try:
            return int(str(installs).replace(',', '').replace('+', '').strip())
        except ValueError:
            return 0

    @staticmethod
    def clean_android_version(version: str) -> Optional[float]:
        """Estrae e normalizza la versione Android"""
        if pd.isna(version):
            return np.nan
        try:
            match = re.search(r'(\d+\.?\d?)', str(version))
            return round(float(match.group(1)), 1) if match else np.nan
        except (ValueError, AttributeError):
            return np.nan

class AppsDataCleaner(DataCleaner):

    def __init__(self, max_workers: int = 4):
        self.max_workers = max_workers
        self.cleaning_steps = []

    def _parallel_clean_row(self, row: pd.Series) -> pd.Series:
        """Pulisce una singola riga del dataset in parallelo"""
        row = row.copy()
        row['Size_MB'] = self.clean_size(row['Size'])
        row['Price_Clean'] = self.clean_price(row['Price'])
        row['Installs_Clean'] = self.clean_installs(row['Installs'])
        row['Android_Ver_Clean'] = self.clean_android_version(row['Android Ver'])
        return row

    def clean_apps_dataset(self, df: pd.DataFrame) -> Tuple[pd.DataFrame, CleaningReport]:
        """Pulisce il dataset delle applicazioni"""
        logger.info("Pulizia dataset applicazioni in corso...")
        df_clean = df.copy()
        original_rows = len(df_clean)
        missing_before = df_clean.isnull().sum().to_dict()

        # Pulizia parallela delle righe
        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            cleaned_rows = list(executor.map(self._parallel_clean_row, [row for _, row in df_clean.iterrows()]))
        df_clean = pd.DataFrame(cleaned_rows, index=df_clean.index)

        # Conversione dei tipi di dato
        df_clean['Rating'] = pd.to_numeric(df_clean['Rating'], errors='coerce')
        df_clean['Reviews'] = pd.to_numeric(df_clean['Reviews'], errors='coerce')
        df_clean['Last Updated'] = pd.to_datetime(df_clean['Last Updated'], errors='coerce')

        # Rimozione righe con valori mancanti critici
        df_clean = df_clean.dropna(subset=['Android_Ver_Clean'])

        # Feature engineering
        df_clean['Days_Since_Update'] = (pd.Timestamp.now() - df_clean['Last Updated']).dt.days

        # Categorizzazione
        df_clean['Price_Category'] = pd.cut(
            df_clean['Price_Clean'],
            bins=[-np.inf, 0, 0.99, 2.99, 4.99, np.inf],
            labels=['Free', 'Very Low', 'Low', 'Medium', 'Premium']
        )

        df_clean['Install_Category'] = pd.cut(
            df_clean['Installs_Clean'],
            bins=[0, 1000, 100000, 1000000, 10000000, np.inf],
            labels=['Very Low', 'Low', 'Medium', 'High', 'Very High']
        )

        # Calcolo metriche di mercato
        total_installs = df_clean['Installs_Clean'].sum()
        df_clean['Market_Share'] = df_clean['Installs_Clean'] / total_installs

        df_clean['Category_Share'] = df_clean.groupby('Category')['Installs_Clean'].transform(
            lambda x: x / x.sum()
        )

        # Report pulizia
        cleaning_report = CleaningReport(
            original_rows=original_rows,
            cleaned_rows=len(df_clean),
            removed_rows=original_rows - len(df_clean),
            missing_before=missing_before,
            missing_after=df_clean.isnull().sum().to_dict(),
            cleaning_steps=self.cleaning_steps
        )

        return df_clean, cleaning_report

class ReviewsDataCleaner(DataCleaner):

    def clean_reviews_dataset(self, df: pd.DataFrame) -> Tuple[pd.DataFrame, CleaningReport]:
        """Pulisce il dataset delle recensioni"""
        logger.info("Pulizia dataset recensioni in corso...")
        df_clean = df.copy()
        original_rows = len(df_clean)
        missing_before = df_clean.isnull().sum().to_dict()

        # Pulizia valori mancanti
        df_clean = df_clean.dropna(subset=['Sentiment', 'Sentiment_Polarity'])

        # Conversione tipi di dato
        df_clean['Sentiment_Polarity'] = pd.to_numeric(df_clean['Sentiment_Polarity'], errors='coerce')
        df_clean['Sentiment_Subjectivity'] = pd.to_numeric(df_clean['Sentiment_Subjectivity'], errors='coerce')

        # Feature engineering
        df_clean['Review_Length'] = df_clean['Translated_Review'].str.len()

        # Categorizzazione sentiment
        df_clean['Sentiment_Category'] = pd.cut(
            df_clean['Sentiment_Polarity'],
            bins=[-1, -0.33, 0.33, 1],
            labels=['Negative', 'Neutral', 'Positive']
        )

        # Report pulizia
        cleaning_report = CleaningReport(
            original_rows=original_rows,
            cleaned_rows=len(df_clean),
            removed_rows=original_rows - len(df_clean),
            missing_before=missing_before,
            missing_after=df_clean.isnull().sum().to_dict(),
            cleaning_steps=[]
        )

        return df_clean, cleaning_report

def clean_datasets(apps_df: pd.DataFrame,
                  reviews_df: pd.DataFrame,
                  apps_cleaner: Optional[AppsDataCleaner] = None,
                  reviews_cleaner: Optional[ReviewsDataCleaner] = None) -> Tuple[pd.DataFrame, pd.DataFrame]:
    logger.info("=== INIZIO PROCESSO DI PULIZIA ===")

    # Inizializzazione cleaners se non forniti
    apps_cleaner = apps_cleaner or AppsDataCleaner()
    reviews_cleaner = reviews_cleaner or ReviewsDataCleaner()

    try:
        # Pulizia dataset app
        apps_clean, apps_report = apps_cleaner.clean_apps_dataset(apps_df)
        logger.info(f"\nPulizia apps dataset completata:")
        logger.info(f"Righe originali: {apps_report.original_rows:,}")
        logger.info(f"Righe dopo pulizia: {apps_report.cleaned_rows:,}")
        logger.info(f"Righe rimosse: {apps_report.removed_rows:,}")

        # Pulizia dataset recensioni
        reviews_clean, reviews_report = reviews_cleaner.clean_reviews_dataset(reviews_df)
        logger.info(f"\nPulizia reviews dataset completata:")
        logger.info(f"Righe originali: {reviews_report.original_rows:,}")
        logger.info(f"Righe dopo pulizia: {reviews_report.cleaned_rows:,}")
        logger.info(f"Righe rimosse: {reviews_report.removed_rows:,}")

        return apps_clean, reviews_clean

    except Exception as e:
        logger.error(f"Errore durante la pulizia dei dati: {str(e)}")
        raise

# Esecuzione della pulizia
apps_clean, reviews_clean = clean_datasets(apps_df, reviews_df)

logger = logging.getLogger(__name__)

@dataclass
class CategoryStats:
    num_apps: int
    avg_rating: float
    paid_perc: float
    avg_installs: float
    avg_size: float

@dataclass
class MarketAnalysis:
    category_stats: pd.DataFrame
    price_stats: Dict[str, float]
    market_analysis: pd.DataFrame
    figures: List[go.Figure] = field(default_factory=list)

class MarketAnalyzer:

    def __init__(self, apps_df: pd.DataFrame, max_workers: int = 4):
        self.apps_df = apps_df[apps_df['Category'] != '1.9'].copy()
        self.max_workers = max_workers

    @lru_cache(maxsize=None)
    def _calculate_category_stats(self, category: str) -> CategoryStats:
        cat_data = self.apps_df[self.apps_df['Category'] == category]
        return CategoryStats(
            num_apps=len(cat_data),
            avg_rating=cat_data['Rating'].mean(),
            paid_perc=(cat_data['Price_Clean'] > 0).mean() * 100,
            avg_installs=cat_data['Installs_Clean'].mean(),
            avg_size=cat_data['Size_MB'].mean()
        )

    def analyze_category_distribution(self) -> Tuple[pd.DataFrame, go.Figure]:
        # Calcolo parallelo delle statistiche per categoria
        categories = self.apps_df['Category'].unique()
        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            stats = list(executor.map(self._calculate_category_stats, categories))

        # Creazione DataFrame
        category_stats = pd.DataFrame({
            'Category': categories,
            'num_apps': [s.num_apps for s in stats],
            'avg_rating': [s.avg_rating for s in stats],
            'paid_perc': [s.paid_perc for s in stats],
            'avg_installs': [s.avg_installs for s in stats],
            'avg_size': [s.avg_size for s in stats]
        }).round(2)

        # Creazione grafico ottimizzato
        fig = px.bar(
            category_stats,
            x='Category',
            y='num_apps',
            color='avg_rating',
            title='Distribuzione delle app per categoria e rating medio',
            labels={
                'num_apps': 'Numero di app',
                'Category': 'Categoria',
                'avg_rating': 'Rating medio'
            },
            color_continuous_scale=[[0, '#B30000'], [0.4, '#FF0000'],
                                  [0.6, '#FFA500'], [0.75, '#2ECC40'],
                                  [1, '#00B300']],
            range_color=[3.2, 4.8]
        )

        fig.update_layout(
            xaxis_tickangle=-45,
            showlegend=True,
            height=600,
            title_x=0.5,
            font=dict(family="Arial", size=12),
            margin=dict(t=100, l=50, r=50, b=100)
        )

        return category_stats.set_index('Category'), fig

    def analyze_price_distribution(self) -> Tuple[Dict[str, float], go.Figure]:
        df_paid = self.apps_df[self.apps_df['Price_Clean'] > 0].copy()

        price_ranges = [0, 1, 2.99, 4.99, 9.99, float('inf')]
        price_labels = ['0-1$', '1-2.99$', '3-4.99$', '5-9.99$', '10$+']

        df_paid['price_range'] = pd.cut(df_paid['Price_Clean'],
                                      bins=price_ranges,
                                      labels=price_labels)

        # Calcolo statistiche prezzi
        price_stats = {
            'total_apps': len(self.apps_df),
            'paid_apps': len(df_paid),
            'paid_percentage': (len(df_paid) / len(self.apps_df)) * 100,
            'avg_price': df_paid['Price_Clean'].mean(),
            'median_price': df_paid['Price_Clean'].median(),
            'max_price': df_paid['Price_Clean'].max()
        }

        # Aggregazione dati per il grafico
        price_distribution = df_paid.groupby('price_range').agg({
            'App': 'count',
            'Rating': 'mean'
        }).reset_index()

        price_distribution['percentage'] = (
            price_distribution['App'] / len(df_paid)
        ) * 100

        # Creazione grafico
        fig = go.Figure(data=[
            go.Bar(
                x=price_distribution['price_range'],
                y=price_distribution['percentage'],
                marker=dict(
                    color=price_distribution['Rating'],
                    colorscale=[[0, '#B30000'], [0.4, '#FF0000'],
                               [0.6, '#FFA500'], [0.75, '#2ECC40'],
                               [1, '#00B300']],
                    colorbar=dict(
                        title="Rating medio",
                        titleside="right",
                        xpad=30,
                        len=0.9,
                        thickness=20
                    ),
                    cmin=3.2,
                    cmax=4.8
                ),
                text=price_distribution['percentage'].round(1).astype(str) + '%',
                textposition='outside'
            )
        ])

        fig.update_layout(
            title='Distribuzione dei prezzi delle app a pagamento',
            title_x=0.5,
            xaxis_title='Fascia di prezzo',
            yaxis_title='Percentuale di app (%)',
            height=500,
            yaxis_range=[0, max(price_distribution['percentage']) * 1.1],
            bargap=0.2,
            font=dict(family="Arial", size=12)
        )

        return price_stats, fig

    def analyze_market_competition(self) -> Tuple[pd.DataFrame, go.Figure]:
        """Analizza la competitività del mercato"""
        market_analysis = self.apps_df.groupby('Category').agg({
            'App': 'count',
            'Rating': 'mean',
            'Price_Clean': lambda x: (x > 0).mean() * 100,
            'Installs_Clean': 'mean'
        }).round(2)

        market_analysis.columns = ['num_apps', 'avg_rating', 'paid_perc', 'avg_installs']

        # Calcolo indice dimensione mercato
        market_analysis['market_size_index'] = (
            (market_analysis['num_apps'] - market_analysis['num_apps'].min()) /
            (market_analysis['num_apps'].max() - market_analysis['num_apps'].min())
        )

        # Creazione grafico
        fig = go.Figure(data=[
            go.Scatter(
                x=market_analysis['num_apps'],
                y=market_analysis['avg_rating'],
                mode='markers+text',
                text=market_analysis.index,
                textposition='top center',
                marker=dict(
                    size=market_analysis['market_size_index'] * 50,
                    color=market_analysis['avg_rating'],
                    colorscale=[[0, '#B30000'], [0.4, '#FF0000'],
                               [0.6, '#FFA500'], [0.75, '#2ECC40'],
                               [1, '#00B300']],
                    colorbar=dict(
                        title="Rating medio",
                        titleside="right",
                        xpad=30,
                        len=0.9,
                        thickness=20
                    ),
                    cmin=3.2,
                    cmax=4.8
                )
            )
        ])

        fig.update_layout(
            title='Mappa competitiva del mercato',
            title_x=0.5,
            xaxis_title='Numero di app (competizione)',
            yaxis_title='Rating medio',
            height=600,
            showlegend=False,
            font=dict(family="Arial", size=12)
        )

        # Calcolo opportunity score
        market_analysis['opportunity_score'] = (
            market_analysis['avg_rating'] * 0.4 +
            (1 - market_analysis['market_size_index']) * 0.3 +
            market_analysis['paid_perc'] * 0.3
        )

        # Aggiungi annotazione con migliori opportunità
        top_opportunities = market_analysis.nlargest(5, 'opportunity_score')
        top_text = "<b>Top 5 opportunità di mercato:</b><br>"
        for cat in top_opportunities.index:
            score = market_analysis.loc[cat, 'opportunity_score']
            rating = market_analysis.loc[cat, 'avg_rating']
            apps = market_analysis.loc[cat, 'num_apps']
            paid = market_analysis.loc[cat, 'paid_perc']
            top_text += f"<b>{cat}</b>: score {score:.2f} (rating {rating:.1f}, app {apps}, propensione {paid:.1f}%)<br>"

        fig.add_annotation(
            x=0.99,
            y=0.01,
            xref="paper",
            yref="paper",
            xanchor="right",
            yanchor="bottom",
            text=top_text,
            showarrow=False,
            font=dict(size=11),
            align="left",
            bgcolor="rgba(255, 255, 255, 0.9)",
            bordercolor="black",
            borderwidth=1,
            borderpad=6
        )

        return market_analysis, fig

def perform_exploratory_analysis(apps_df: pd.DataFrame,
                               reviews_df: Optional[pd.DataFrame] = None,
                               analyzer: Optional[MarketAnalyzer] = None) -> MarketAnalysis:
    logger.info("=== INIZIO ANALISI ESPLORATIVA ===")

    analyzer = analyzer or MarketAnalyzer(apps_df)
    figures = []

    try:
        # 1. Analisi distribuzione per categoria
        logger.info("\n1. Analisi distribuzione per categoria")
        category_stats, category_fig = analyzer.analyze_category_distribution()
        figures.append(category_fig)

        # 2. Analisi prezzi
        logger.info("\n2. Analisi distribuzione prezzi")
        price_stats, price_fig = analyzer.analyze_price_distribution()
        figures.append(price_fig)

        # 3. Analisi competitiva
        logger.info("\n3. Analisi competitiva del mercato")
        market_analysis, market_fig = analyzer.analyze_market_competition()
        figures.append(market_fig)

        # Log risultati principali
        logger.info("\nTop 5 opportunità di mercato:")
        top_opportunities = market_analysis.nlargest(5, 'opportunity_score')
        for cat in top_opportunities.index:
            logger.info(f"\n{cat}:")
            logger.info(f"- Score: {market_analysis.loc[cat, 'opportunity_score']:.2f}")
            logger.info(f"- Rating medio: {market_analysis.loc[cat, 'avg_rating']:.2f}")
            logger.info(f"- Competizione: {market_analysis.loc[cat, 'num_apps']:,} app")
            logger.info(f"- % app a pagamento: {market_analysis.loc[cat, 'paid_perc']:.1f}%")

        return MarketAnalysis(
            category_stats=category_stats,
            price_stats=price_stats,
            market_analysis=market_analysis,
            figures=figures
        )

    except Exception as e:
        logger.error(f"Errore durante l'analisi esplorativa: {str(e)}")
        raise

# Esecuzione dell'analisi esplorativa
market_analysis = perform_exploratory_analysis(apps_clean, reviews_clean)

# Visualizzazione dei grafici
for fig in market_analysis.figures:
    fig.show()

logger = logging.getLogger(__name__)

class CorrelationResults(NamedTuple):
    pearson: pd.DataFrame
    spearman: pd.DataFrame
    figures: List[go.Figure]

class TimeMetrics(NamedTuple):
    data: pd.DataFrame
    figure: go.Figure

@dataclass
class PlayStoreAnalyzer:

    apps_df: pd.DataFrame
    reviews_df: Optional[pd.DataFrame] = None
    max_workers: int = 4

    def __post_init__(self):
        self.apps_df = self.apps_df[self.apps_df['Category'] != '1.9'].copy()
        self.prepare_data()

    def prepare_data(self) -> None:

        # Metriche temporali
        self.apps_df['Last Updated'] = pd.to_datetime(self.apps_df['Last Updated'])

        # Usa la data più recente nel dataset come riferimento
        max_date = self.apps_df['Last Updated'].max()
        self.apps_df['Days_Since_Update'] = (
            max_date - self.apps_df['Last Updated']
        ).dt.days

        self.apps_df['Update_Year'] = self.apps_df['Last Updated'].dt.year

        # Trasformazioni logaritmiche
        self.apps_df['Log_Installs'] = np.log1p(self.apps_df['Installs_Clean'])
        self.apps_df['Log_Size'] = np.log1p(self.apps_df['Size_MB'])

        # Metriche di mercato
        total_installs = self.apps_df['Installs_Clean'].sum()
        self.apps_df['market_share'] = self.apps_df['Installs_Clean'] / total_installs
        self.apps_df['category_share'] = self.apps_df.groupby('Category')['Installs_Clean'].transform(
            lambda x: x / x.sum()
        )

        # Merge con sentiment se disponibile
        if self.reviews_df is not None:
            self._merge_sentiment_data()

    def _merge_sentiment_data(self) -> None:
        sentiment_data = self.reviews_df.merge(
            self.apps_df[['App', 'Category']],
            on='App',
            how='inner'
        )

        app_sentiment = sentiment_data.groupby(['App', 'Category']).agg({
            'Sentiment_Polarity': 'mean',
            'Sentiment_Subjectivity': 'mean'
        }).reset_index()

        self.apps_df = self.apps_df.merge(
            app_sentiment,
            on=['App', 'Category'],
            how='left'
        )

    @staticmethod
    def _create_correlation_heatmap(corr_matrix: pd.DataFrame,
                                  title: str) -> go.Figure:
        return go.Figure(
            data=go.Heatmap(
                z=corr_matrix.values,
                x=corr_matrix.columns,
                y=corr_matrix.index,
                colorscale='RdBu',
                zmin=-1,
                zmax=1,
                text=corr_matrix.values.round(2),
                texttemplate='%{text}',
                textfont={"size": 10}
            ),
            layout=dict(
                title=title,
                height=600,
                font=dict(family="Arial", size=12)
            )
        )

    def _create_scatter_matrix(self, metrics: Dict[str, str]) -> go.Figure:
        scatter_pairs = [
            ('Rating', 'Log_Installs'),
            ('Rating', 'Log_Size'),
            ('Log_Size', 'Log_Installs'),
            ('Rating', 'Days_Since_Update')
        ]

        fig = make_subplots(
            rows=2, cols=2,
            subplot_titles=[
                f'{metrics[x]} vs {metrics[y]}'
                for x, y in scatter_pairs
            ]
        )

        # Creazione di una scala di colori per i giorni
        colorscale = [
            [0, 'green'],      # 0-30 giorni
            [0.1, 'lightgreen'],  # 30-90 giorni
            [0.2, 'yellow'],   # 90-180 giorni
            [0.4, 'orange'],   # 6 mesi-1 anno
            [0.6, 'red'],      # 1-2 anni
            [1.0, 'darkred']   # >2 anni
        ]

        for idx, (x, y) in enumerate(scatter_pairs):
            row = idx // 2 + 1
            col = idx % 2 + 1

            hover_text = [
                f"App: {app}<br>" +
                f"Categoria: {cat}<br>" +
                f"{metrics[x]}: {val_x:.2f}<br>" +
                f"{metrics[y]}: {val_y:.2f}<br>" +
                f"Prezzo: ${price:.2f}<br>" +
                f"Installazioni: {inst:,.0f}<br>" +
                f"Dimensione: {size:.1f}MB<br>" +
                f"Giorni dall'ultimo aggiornamento: {days:.0f}"
                for app, cat, val_x, val_y, price, inst, size, days in zip(
                    self.apps_df['App'],
                    self.apps_df['Category'],
                    self.apps_df[x],
                    self.apps_df[y],
                    self.apps_df['Price_Clean'],
                    self.apps_df['Installs_Clean'],
                    self.apps_df['Size_MB'],
                    self.apps_df['Days_Since_Update']
                )
            ]

            fig.add_trace(
                go.Scatter(
                    x=self.apps_df[x],
                    y=self.apps_df[y],
                    mode='markers',
                    marker=dict(
                        size=4,
                        opacity=0.6,
                        color=self.apps_df['Days_Since_Update'],
                        colorscale=colorscale,
                        colorbar=dict(
                            title='Giorni dall\'ultimo<br>aggiornamento',
                            ticktext=['0', '30', '90', '180', '365', '730', '>730'],
                            tickvals=[0, 30, 90, 180, 365, 730, 1000]
                        ) if idx == 1 else None,
                        cmin=0,
                        cmax=1000
                    ),
                    name=f'{metrics[x]} vs {metrics[y]}',
                    hovertemplate="%{text}<extra></extra>",
                    text=hover_text,
                    showlegend=False
                ),
                row=row, col=col
            )

            # Aggiornamento degli assi
            fig.update_xaxes(title=metrics[x], row=row, col=col, gridcolor='lightgray', showgrid=True)
            fig.update_yaxes(title=metrics[y], row=row, col=col, gridcolor='lightgray', showgrid=True)

        fig.update_layout(
            title='Scatter plots delle relazioni principali',
            height=800,
            width=1000,
            showlegend=False,
            title_x=0.5,
            hovermode='closest',
            plot_bgcolor='white',
            margin=dict(t=100, l=50, r=50, b=50)
        )

        return fig

    def analyze_correlations(self) -> Tuple[pd.DataFrame, pd.DataFrame, List[go.Figure]]:
        metrics = {
            'Rating': 'Rating',
            'Price_Clean': 'Prezzo',
            'Log_Installs': 'Log installazioni',
            'Log_Size': 'Log dimensione',
            'Days_Since_Update': 'Giorni da ultimo aggiornamento'
        }

        # Calcolo correlazioni
        data = self.apps_df[metrics.keys()]
        pearson_corr = data.corr(method='pearson').round(3)
        spearman_corr = data.corr(method='spearman').round(3)

        # Rinomina colonne
        for corr_matrix in [pearson_corr, spearman_corr]:
            corr_matrix.columns = metrics.values()
            corr_matrix.index = metrics.values()

        # Creazione grafici
        figures = []

        # Heatmap correlazioni
        heatmap_fig = make_subplots(
            rows=1, cols=2,
            subplot_titles=('Correlazioni di Pearson', 'Correlazioni di Spearman'),
            horizontal_spacing=0.15
        )

        # Aggiungi heatmap Pearson
        heatmap_fig.add_trace(
            go.Heatmap(
                z=pearson_corr.values,
                x=pearson_corr.columns,
                y=pearson_corr.index,
                colorscale='RdBu',
                zmin=-1,
                zmax=1,
                text=pearson_corr.values.round(2),
                texttemplate='%{text}',
                textfont={"size": 10}
            ),
            row=1, col=1
        )

        # Aggiungi heatmap Spearman
        heatmap_fig.add_trace(
            go.Heatmap(
                z=spearman_corr.values,
                x=spearman_corr.columns,
                y=spearman_corr.index,
                colorscale='RdBu',
                zmin=-1,
                zmax=1,
                text=spearman_corr.values.round(2),
                texttemplate='%{text}',
                textfont={"size": 10}
            ),
            row=1, col=2
        )

        heatmap_fig.update_layout(
            title='Confronto correlazioni Pearson vs Spearman',
            title_x=0.5,
            height=600,
            width=1500,  # Aumentato a 1500
            font=dict(family="Arial", size=12),
            margin=dict(t=100, l=100, r=100, b=50)
        )

        figures.append(heatmap_fig)

        # Scatter matrix
        scatter_pairs = [
            ('Rating', 'Log_Installs'),
            ('Rating', 'Log_Size'),
            ('Log_Size', 'Log_Installs'),
            ('Rating', 'Days_Since_Update')
        ]

        scatter_fig = make_subplots(
            rows=2, cols=2,
            subplot_titles=[
                f'{metrics[x]} vs {metrics[y]}'
                for x, y in scatter_pairs
            ],
            horizontal_spacing=0.15,
            vertical_spacing=0.15
        )

        # Creazione di una scala di colori basata sul rating
        colorscale = [
            [0, 'red'],       # Rating 1
            [0.25, 'orange'], # Rating 2
            [0.5, 'yellow'],  # Rating 3
            [0.75, 'lightgreen'], # Rating 4
            [1, 'green']      # Rating 5
        ]

        for idx, (x, y) in enumerate(scatter_pairs):
            row = idx // 2 + 1
            col = idx % 2 + 1

            hover_text = [
                f"App: {app}<br>" +
                f"Categoria: {cat}<br>" +
                f"{metrics[x]}: {val_x:.2f}<br>" +
                f"{metrics[y]}: {val_y:.2f}<br>" +
                f"Rating: {rating:.1f}<br>" +
                f"Prezzo: ${price:.2f}<br>" +
                f"Installazioni: {inst:,.0f}<br>" +
                f"Dimensione: {size:.1f}MB<br>" +
                f"Giorni dall'ultimo aggiornamento: {days:.0f}"
                for app, cat, val_x, val_y, rating, price, inst, size, days in zip(
                    self.apps_df['App'],
                    self.apps_df['Category'],
                    self.apps_df[x],
                    self.apps_df[y],
                    self.apps_df['Rating'],
                    self.apps_df['Price_Clean'],
                    self.apps_df['Installs_Clean'],
                    self.apps_df['Size_MB'],
                    self.apps_df['Days_Since_Update']
                )
            ]

            if y == 'Days_Since_Update':
                # Aggiunge jitter al rating per evitare sovrapposizioni
                jittered_x = self.apps_df[x] + np.random.normal(0, 0.05, len(self.apps_df))

                # Calcola la media mobile
                rating_range = np.arange(1, 5.1, 0.1)
                days_mean = []
                for r in rating_range:
                    mask = (self.apps_df[x] >= r - 0.2) & (self.apps_df[x] < r + 0.2)
                    mean_val = self.apps_df.loc[mask, y].mean()
                    days_mean.append(mean_val)

                scatter_fig.add_trace(
                    go.Scatter(
                        x=jittered_x,
                        y=self.apps_df[y],
                        mode='markers',
                        marker=dict(
                            size=3,
                            opacity=0.3,
                            color=self.apps_df['Rating'],
                            colorscale=colorscale,
                            colorbar=dict(
                                title='Rating',
                                ticktext=['1', '2', '3', '4', '5'],
                                tickvals=[1, 2, 3, 4, 5]
                            ) if idx == 1 else None,
                            cmin=1,
                            cmax=5
                        ),
                        name=f'{metrics[x]} vs {metrics[y]}',
                        hovertemplate="%{text}<extra></extra>",
                        text=hover_text,
                        showlegend=False
                    ),
                    row=row, col=col
                )

                # Aggiunge la linea della media mobile
                scatter_fig.add_trace(
                    go.Scatter(
                        x=rating_range,
                        y=days_mean,
                        mode='lines',
                        line=dict(color='black', width=2),
                        name='Media mobile',
                        showlegend=False
                    ),
                    row=row, col=col
                )

                # Aggiorna il layout per questo subplot specifico
                scatter_fig.update_xaxes(
                    title=metrics[x],
                    row=row,
                    col=col,
                    gridcolor='lightgray',
                    showgrid=True,
                    range=[0.5, 5.5]
                )

                # Usa il 95° percentile per l'asse y
                y_max = self.apps_df[y].quantile(0.95)
                scatter_fig.update_yaxes(
                    title=metrics[y],
                    row=row,
                    col=col,
                    gridcolor='lightgray',
                    showgrid=True,
                    range=[0, y_max]
                )
            else:
                scatter_fig.add_trace(
                    go.Scatter(
                        x=self.apps_df[x],
                        y=self.apps_df[y],
                        mode='markers',
                        marker=dict(
                            size=4,
                            opacity=0.6,
                            color=self.apps_df['Rating'],
                            colorscale=colorscale,
                            colorbar=dict(
                                title='Rating',
                                ticktext=['1', '2', '3', '4', '5'],
                                tickvals=[1, 2, 3, 4, 5]
                            ) if idx == 1 else None,
                            cmin=1,
                            cmax=5
                        ),
                        name=f'{metrics[x]} vs {metrics[y]}',
                        hovertemplate="%{text}<extra></extra>",
                        text=hover_text,
                        showlegend=False
                    ),
                    row=row, col=col
                )

                scatter_fig.update_xaxes(
                    title=metrics[x],
                    row=row,
                    col=col,
                    gridcolor='lightgray',
                    showgrid=True
                )
                scatter_fig.update_yaxes(
                    title=metrics[y],
                    row=row,
                    col=col,
                    gridcolor='lightgray',
                    showgrid=True
                )

        scatter_fig.update_layout(
            title='Scatter plots delle relazioni principali',
            height=800,
            width=1500,  # Aumentato a 1500
            showlegend=False,
            title_x=0.5,
            hovermode='closest',
            plot_bgcolor='white',
            margin=dict(t=100, l=50, r=50, b=50)
        )

        figures.append(scatter_fig)

        return pearson_corr, spearman_corr, figures


    def analyze_temporal_trends(self) -> TimeMetrics:
        # Verifica preliminare prezzi
        avg_price = self.apps_df.groupby('Update_Year').agg({
            'Price_Clean': lambda x: x[x > 0].mean() if len(x[x > 0]) > 0 else np.nan
        })

        # Aggregazione metriche temporali efficiente
        time_metrics = self.apps_df.groupby('Update_Year').agg({
            'Rating': 'mean',
            'Size_MB': 'mean',
            'Installs_Clean': 'mean',
            'App': 'count'
        }).round(2)

        time_metrics['Price_Clean'] = avg_price['Price_Clean']

        # Creazione figura ottimizzata con subplot
        fig = make_subplots(
            rows=2,
            cols=1,
            row_heights=[0.6, 0.4],
            vertical_spacing=0.12,
            subplot_titles=(
                'Metriche di prodotto (Rating, Prezzo, Dimensione)',
                'Metriche di mercato (Installazioni e numero di app)'
            )
        )

        # Configurazione tracce subplot 1
        traces_subplot1 = [
            ('Rating', 'Rating medio', '#2ECC40'),
            ('Price_Clean', 'Prezzo medio ($)', '#FF4136'),
            ('Size_MB', 'Dimensione media (MB)', '#0074D9')
        ]

        # Aggiunta tracce subplot 1
        for col, name, color in traces_subplot1:
            data = time_metrics[col].fillna(0)
            hover_text = [
                f"Anno: {year}<br>{name}: {self._format_trend_value(val, col.lower())}"
                for year, val in zip(time_metrics.index, time_metrics[col])
            ]

            fig.add_trace(
                go.Scatter(
                    x=time_metrics.index,
                    y=data,
                    name=name,
                    line=dict(color=color, width=2),
                    hovertemplate="%{text}<extra></extra>",
                    text=hover_text
                ),
                row=1,
                col=1
            )

        # Aggiunta barre numero app subplot 2
        hover_text_app = [
            f"Anno: {year}<br>Numero di app: {self._format_trend_value(val, 'app')}"
            for year, val in zip(time_metrics.index, time_metrics['App'])
        ]

        fig.add_trace(
            go.Bar(
                x=time_metrics.index,
                y=time_metrics['App'],
                name='Numero di app',
                marker_color='#AAAAAA',
                opacity=0.3,
                width=0.5,
                hovertemplate="%{text}<extra></extra>",
                text=hover_text_app
            ),
            row=2,
            col=1
        )

        # Aggiunta linea installazioni subplot 2
        hover_text_inst = [
            f"Anno: {year}<br>Installazioni medie: {self._format_trend_value(val, 'installazioni')}"
            for year, val in zip(time_metrics.index, time_metrics['Installs_Clean'])
        ]

        fig.add_trace(
            go.Scatter(
                x=time_metrics.index,
                y=time_metrics['Installs_Clean'],
                name='Installazioni medie',
                line=dict(color='#FF851B', width=2),
                hovertemplate="%{text}<extra></extra>",
                text=hover_text_inst
            ),
            row=2,
            col=1
        )

        # Ottimizzazione layout
        fig.update_layout(
            title={
                'text': 'Evoluzione temporale delle metriche chiave',
                'y': 0.98,
                'x': 0.5,
                'xanchor': 'center',
                'yanchor': 'top'
            },
            height=900,
            showlegend=True,
            legend=dict(
                orientation='h',
                yanchor='bottom',
                y=1.05,
                xanchor='center',
                x=0.5,
                bgcolor='rgba(255, 255, 255, 0.8)',
                bordercolor='lightgray',
                borderwidth=1
            ),
            plot_bgcolor='white',
            hovermode='x unified',
            margin=dict(t=120, b=50, l=50, r=50)
        )

        # Ottimizzazione assi
        for row in [1, 2]:
            fig.update_xaxes(
                title='Anno',
                showgrid=True,
                gridwidth=1,
                gridcolor='lightgray',
                row=row
            )

        fig.update_yaxes(
            title='Valore',
            showgrid=True,
            gridwidth=1,
            gridcolor='lightgray',
            row=1,
            col=1
        )

        fig.update_yaxes(
            title='Numero',
            showgrid=True,
            gridwidth=1,
            gridcolor='lightgray',
            type='log',
            row=2,
            col=1
        )

        return TimeMetrics(time_metrics, fig)

    def _format_trend_value(self, value: float, metric_type: str) -> str:
        if pd.isna(value):
            return "N/D"

        if metric_type == "installazioni":
            if value >= 1e6:
                return f"{value/1e6:.1f}M"
            elif value >= 1e3:
                return f"{value/1e3:.1f}K"
            return f"{value:.0f}"
        elif metric_type == "dimensione":
            return f"{value:.1f}MB"
        elif metric_type == "prezzo":
            return f"${value:.2f}"
        elif metric_type == "rating":
            return f"{value:.2f}"
        elif metric_type == "app":
            return f"{int(value):,}"
        return str(value)

def analyze_play_store(apps_df: pd.DataFrame, reviews_df: Optional[pd.DataFrame] = None) -> Dict[str, Any]:
    logger.info("=== ANALISI GOOGLE PLAY STORE ===")

    # Inizializzazione analyzer
    analyzer = PlayStoreAnalyzer(apps_df, reviews_df)
    figures = []
    results = {}

    try:
        # 1. Analisi correlazioni
        logger.info("\n1. Analisi correlazioni tra metriche")
        pearson_corr, spearman_corr, corr_figures = analyzer.analyze_correlations()
        results['correlations'] = {'pearson': pearson_corr, 'spearman': spearman_corr}
        figures.extend(corr_figures)

        # Log correlazioni
        for method, corr_matrix in [('Pearson', pearson_corr), ('Spearman', spearman_corr)]:
            logger.info(f"\nCorrelazioni {method} significative (|corr| > 0.3):")
            for i in range(len(corr_matrix.columns)):
                for j in range(i+1, len(corr_matrix.columns)):
                    corr = corr_matrix.iloc[i, j]
                    if abs(corr) > 0.3:
                        logger.info(
                            f"{corr_matrix.index[i]} vs {corr_matrix.columns[j]}: {corr:.3f}"
                        )

        # 2. Analisi temporale
        logger.info("\n2. Analisi trend temporali")
        time_metrics, time_fig = analyzer.analyze_temporal_trends()
        results['temporal'] = time_metrics
        figures.append(time_fig)

        # Log trend principali
        first_metrics = time_metrics.iloc[0]
        last_metrics = time_metrics.iloc[-1]

        rating_change = ((last_metrics['Rating'] - first_metrics['Rating']) /
                        first_metrics['Rating'] * 100)

        logger.info("\nTrend principali:")
        logger.info(
            f"Rating medio: {rating_change:+.1f}% di variazione "
            f"(da {first_metrics['Rating']:.2f} a {last_metrics['Rating']:.2f})"
        )
        logger.info(
            f"Dimensione media: da {first_metrics['Size_MB']:.1f}MB a "
            f"{last_metrics['Size_MB']:.1f}MB"
        )

        def format_installs(val):
            return f"{val/1e6:.1f}M" if val >= 1e6 else f"{val/1e3:.1f}K"

        logger.info(
            f"Installazioni medie: da {format_installs(first_metrics['Installs_Clean'])} a "
            f"{format_installs(last_metrics['Installs_Clean'])}"
        )

        logger.info(
            f"Numero di app: da {int(first_metrics['App']):,} a "
            f"{int(last_metrics['App']):,}"
        )

        if pd.notna(first_metrics['Price_Clean']) and pd.notna(last_metrics['Price_Clean']):
            logger.info(
                f"Prezzo medio: da ${first_metrics['Price_Clean']:.2f} a "
                f"${last_metrics['Price_Clean']:.2f}"
            )
        else:
            logger.info("Prezzo medio: dati non disponibili")

        results['figures'] = figures
        return results

    except Exception as e:
        logger.error(f"Errore durante l'analisi del Play Store: {str(e)}")
        raise

# Esecuzione dell'analisi
analysis_results = analyze_play_store(apps_clean, reviews_clean)

# Visualizzazione dei grafici
for fig in analysis_results['figures']:
    fig.show()

logger = logging.getLogger(__name__)

class MarketStructureResults(NamedTuple):
    market_df: pd.DataFrame
    figure: go.Figure

class MarketAnalyzer:

    def __init__(self, apps_df: pd.DataFrame, max_workers: int = 4):
        self.apps_df = apps_df[apps_df['Category'] != '1.9'].copy()
        self.max_workers = max_workers

    @staticmethod
    def calculate_market_concentration(category_data: pd.DataFrame) -> float:
        try:
            if len(category_data) == 0:
                return np.nan

            total_installs = category_data['Installs_Clean'].sum()
            if total_installs == 0:
                return np.nan

            market_shares = category_data['Installs_Clean'] / total_installs
            hhi = (market_shares ** 2).sum()

            return np.clip(hhi, 0, 1)
        except:
            return np.nan

    @staticmethod
    def calculate_category_stability(category_data: pd.DataFrame) -> float:
        try:
            if len(category_data) < 2:
                return np.nan

            # Calcolo variabilità rating
            rating_var = (category_data['Rating'].std() / 5
                         if not category_data['Rating'].isna().all() else 0)

            # Calcolo variabilità aggiornamenti
            update_var = (category_data['Days_Since_Update'].std() / 365
                         if not category_data['Days_Since_Update'].isna().all() else 0)

            # Calcolo variabilità installazioni
            installs = np.log1p(category_data['Installs_Clean'])
            install_var = (installs.std() / installs.mean()
                          if not installs.isna().all() and installs.mean() > 0 else 0)

            # Calcolo stabilità complessiva
            stability = 1 - (rating_var * 0.4 + update_var * 0.3 + install_var * 0.3)
            return np.clip(stability, 0, 1)
        except:
            return np.nan

    @staticmethod
    def calculate_direct_payment_propensity(category_data: pd.DataFrame) -> float:
        try:
            if len(category_data) == 0:
                return np.nan

            # Calcolo ratio app a pagamento
            paid_apps = category_data[category_data['Price_Clean'] > 0]
            paid_ratio = len(paid_apps) / len(category_data)

            # Calcolo livello prezzi
            max_price = category_data['Price_Clean'].max()
            price_level = (paid_apps['Price_Clean'].mean() / max_price
                         if len(paid_apps) > 0 and max_price > 0 else 0)

            # Calcolo differenza rating paid vs free
            if len(paid_apps) > 0:
                free_rating = category_data[category_data['Price_Clean'] == 0]['Rating'].mean()
                paid_rating = paid_apps['Rating'].mean()
                paid_vs_free = (paid_rating - free_rating + 5) / 10
            else:
                paid_vs_free = 0

            # Calcolo propensione complessiva
            propensity = (paid_ratio * 0.4 + price_level * 0.3 + paid_vs_free * 0.3)
            return np.clip(propensity, 0, 1)
        except:
            return np.nan

    def analyze_market_structure(self) -> MarketStructureResults:

        # Lista per raccogliere risultati
        results = []

        # Analisi per categoria
        for category in self.apps_df['Category'].unique():
            category_data = self.apps_df[self.apps_df['Category'] == category]

            results.append({
                'Category': category,
                'num_apps': len(category_data),
                'concentration': self.calculate_market_concentration(category_data),
                'stability': self.calculate_category_stability(category_data),
                'payment_propensity': self.calculate_direct_payment_propensity(category_data),
                'avg_rating': category_data['Rating'].mean(),
                'total_installs': category_data['Installs_Clean'].sum()
            })

        # Creazione DataFrame risultati
        market_df = pd.DataFrame(results).set_index('Category')

        # Creazione grafico
        fig = go.Figure(data=[
            go.Scatter(
                x=market_df['concentration'],
                y=market_df['stability'],
                mode='markers+text',
                text=market_df.index,
                textposition='top right',
                textfont=dict(size=9),
                marker=dict(
                    size=market_df['payment_propensity'].fillna(0) * 50 + 20,
                    color=market_df['avg_rating'].fillna(market_df['avg_rating'].mean()),
                    colorscale='RdYlGn',
                    colorbar=dict(title='Rating medio'),
                    showscale=True
                ),
                hovertemplate=(
                    "<b>%{text}</b><br>" +
                    "Concentrazione: %{x:.3f}<br>" +
                    "Stabilità: %{y:.3f}<br>" +
                    "Propensione pagamento: %{marker.size:.3f}<br>" +
                    "Rating: %{marker.color:.2f}<br>" +
                    "<extra></extra>"
                )
            )
        ])

        # Aggiornamento layout
        fig.update_layout(
            title=dict(
                text='Struttura del mercato per categoria',
                x=0.5,
                y=0.95,
                xanchor='center',
                yanchor='top',
                font=dict(size=20)
            ),
            xaxis_title='Indice di concentrazione del mercato',
            yaxis_title='Indice di stabilità',
            height=800,
            showlegend=False,
            plot_bgcolor='white',
            margin=dict(t=150),
            annotations=[
                dict(
                    text='Dimensione = propensione al pagamento diretto',
                    xref='paper',
                    yref='paper',
                    x=0.5,
                    y=1.08,
                    xanchor='center',
                    yanchor='middle',
                    showarrow=False,
                    font=dict(size=11)
                ),
                dict(
                    text='Colore = rating medio',
                    xref='paper',
                    yref='paper',
                    x=0.5,
                    y=1.04,
                    xanchor='center',
                    yanchor='middle',
                    showarrow=False,
                    font=dict(size=11)
                )
            ]
        )

        return MarketStructureResults(market_df, fig)


class TechnicalAnalyzer:

    def __init__(self, apps_df: pd.DataFrame, max_workers: int = 4):
        self.apps_df = apps_df[apps_df['Category'] != '1.9'].copy()
        self.max_workers = max_workers

    def analyze_development_patterns(self) -> Tuple[pd.DataFrame, List[go.Figure]]:
        """Analizza i pattern di sviluppo per ogni categoria"""
        tech_results = []
        android_distributions = {}

        # Calcolo distribuzioni versioni Android per categoria
        categories = self.apps_df['Category'].unique()
        for category in categories:
            category_data = self.apps_df[self.apps_df['Category'] == category]

            # Distribuzione versioni Android
            version_dist = category_data['Android_Ver_Clean'].value_counts()
            total_apps = len(category_data)
            version_percentages = (version_dist / total_apps * 100).round(1)
            android_distributions[category] = version_percentages

            # Statistiche tecniche
            size_stats = {
                'mean': category_data['Size_MB'].mean(),
                'median': category_data['Size_MB'].median(),
                'std': category_data['Size_MB'].std()
            }

            days_since = category_data['Days_Since_Update'].mean()
            updates_per_year = 365 / days_since if days_since > 0 else np.nan

            tech_results.append({
                'Category': category,
                'avg_size': size_stats['mean'],
                'size_variability': size_stats['std'] / size_stats['mean'] if size_stats['mean'] > 0 else 0,
                'updates_per_year': updates_per_year,
                'num_apps': total_apps
            })

        # Creazione matrice versioni Android
        all_versions = sorted(set().union(*[dist.index for dist in android_distributions.values()]))
        android_matrix = pd.DataFrame(
            index=android_distributions.keys(),
            columns=all_versions,
            data=0.0
        )

        # Popolamento matrice
        for category, dist in android_distributions.items():
            for version in dist.index:
                android_matrix.loc[category, version] = dist[version]

        tech_df = pd.DataFrame(tech_results).set_index('Category')

        # Creazione grafici
        android_fig = self._create_android_distribution_plot(android_matrix)
        tech_fig = self._create_technical_details_plot(tech_df)

        return tech_df, [android_fig, tech_fig]

    def _create_android_distribution_plot(self, android_matrix: pd.DataFrame) -> go.Figure:

        # Ordinamento categorie per versione media
        weighted_avg = pd.DataFrame({
            'avg_version': sum(android_matrix[col] * float(col) for col in android_matrix.columns) / 100
        })
        android_matrix_sorted = android_matrix.loc[weighted_avg.sort_values('avg_version', ascending=False).index]

        # Scala colori per versioni
        n_versions = len(android_matrix_sorted.columns)
        colors = [
            f'rgb({int(255*(1-i/n_versions))}, {int(255*(i/n_versions))}, 0)'
            for i in range(n_versions)
        ]

        # Creazione grafico
        fig = go.Figure()

        for i, version in enumerate(android_matrix_sorted.columns):
            fig.add_trace(go.Bar(
                name=f'Android {version}',
                y=android_matrix_sorted.index,
                x=android_matrix_sorted[version],
                orientation='h',
                marker_color=colors[i],
                hovertemplate=(
                    "<b>%{y}</b><br>" +
                    f"Android {version}: " + "%{x:.1f}%<br>" +
                    "<extra></extra>"
                )
            ))

        fig.update_layout(
            title=dict(
                text='Distribuzione versioni Android per categoria',
                x=0.5,
                font=dict(size=20)
            ),
            xaxis_title='Percentuale di app (%)',
            yaxis_title='Categoria',
            barmode='stack',
            height=800,
            showlegend=True,
            plot_bgcolor='white',
            legend=dict(
                title='Versione Android',
                yanchor="top",
                y=0.99,
                xanchor="left",
                x=1.02
            ),
            margin=dict(l=200, r=150),
            bargap=0.1
        )

        fig.add_vline(x=100, line_dash="dash", line_color="gray", opacity=0.5)

        return fig

    def _create_technical_details_plot(self, tech_df: pd.DataFrame) -> go.Figure:
        categories = tech_df.index.tolist()
        fig = go.Figure()

        for category in categories:
            category_data = self.apps_df[self.apps_df['Category'] == category]
            category_sizes = category_data['Size_MB'].dropna()
            app_names = category_data.loc[category_sizes.index, 'App'].values

            fig.add_trace(go.Box(
                y=category_sizes,
                name=category,
                boxpoints='outliers',
                jitter=0.3,
                pointpos=-1.8,
                hovertemplate=(
                    "<b>%{customdata}</b><br>" +
                    "Categoria: %{x}<br>" +
                    "Dimensione: %{y:.1f}MB<br>" +
                    "<extra></extra>"
                ),
                customdata=app_names
            ))

        fig.update_layout(
            title=dict(
                text='Distribuzione dimensioni app per categoria',
                x=0.5,
                font=dict(size=20)
            ),
            xaxis=dict(
                title='Categoria',
                tickangle=45,
                tickfont=dict(size=10)
            ),
            yaxis_title='Dimensione (MB)',
            showlegend=False,
            height=700,
            margin=dict(b=150, t=150),
            plot_bgcolor='white',
            annotations=[dict(
                text=('Box = range interquartile (25°-75° percentile)<br>' +
                      'Linea = mediana<br>' +
                      'Punti = outlier (app significativamente più pesanti)'),
                xref='paper',
                yref='paper',
                x=0.5,
                y=1.1,
                showarrow=False,
                font=dict(size=12)
            )]
        )

        return fig

def perform_category_analysis(apps_df: pd.DataFrame, reviews_df: pd.DataFrame) -> Dict[str, Any]:
    logger.info("=== ANALISI APPROFONDITA PER CATEGORIA ===\n")
    results = {}
    figures = []

    # 1. Analisi struttura mercato
    logger.info("\n1. Analisi struttura del mercato")
    market_analyzer = MarketAnalyzer(apps_df)
    market_results = market_analyzer.analyze_market_structure()
    results['market'] = market_results.market_df
    figures.append(market_results.figure)

    # Log risultati mercato
    logger.info("\nTop 5 categorie per concentrazione di mercato:")
    print(market_results.market_df[['concentration', 'stability', 'avg_rating']]
          .nlargest(5, 'concentration'))

    # 2. Analisi tecnica
    logger.info("\n2. Analisi aspetti tecnici")
    tech_analyzer = TechnicalAnalyzer(apps_df)
    tech_df, tech_figs = tech_analyzer.analyze_development_patterns()
    results['technical'] = tech_df
    figures.extend(tech_figs)

    # Calcolo score finale
    categories = market_results.market_df.index
    final_scores = pd.DataFrame(index=categories)

    # Market score (80%)
    final_scores['market_score'] = (
        market_results.market_df['stability'] * 0.4 +
        (1 - market_results.market_df['concentration']) * 0.3 +
        market_results.market_df['payment_propensity'].fillna(0) * 0.3
    )

    # Technical score (20%)
    final_scores['technical_score'] = (
        tech_df['updates_per_year'].fillna(0) / tech_df['updates_per_year'].max()
    )

    # Score finale
    final_scores['final_score'] = (
        final_scores['market_score'] * 0.8 +
        final_scores['technical_score'] * 0.2
    )

    results['final_scores'] = final_scores

    # Report finale
    logger.info("\nTop 5 categorie più promettenti:")
    top_5 = final_scores.nlargest(5, 'final_score')
    for cat in top_5.index:
        logger.info(f"\n{cat}:")
        logger.info(f"- Score finale: {top_5.loc[cat, 'final_score']:.3f}")
        logger.info(f"- Score mercato: {top_5.loc[cat, 'market_score']:.3f}")
        logger.info(f"- Score tecnico: {top_5.loc[cat, 'technical_score']:.3f}")
        logger.info(f"- Dettagli mercato:")
        logger.info(f"  * Concentrazione: {market_results.market_df.loc[cat, 'concentration']:.3f}")
        logger.info(f"  * Stabilità: {market_results.market_df.loc[cat, 'stability']:.3f}")
        logger.info(f"  * Propensione al pagamento: {market_results.market_df.loc[cat, 'payment_propensity']:.3f}")

    # Visualizzazione esplicita delle top 5 categorie
    print("\n======== TOP 5 CATEGORIE PIÙ PROMETTENTI ========")
    print(top_5[['final_score', 'market_score', 'technical_score']].round(3))

    print("\nNOTA SULLA METODOLOGIA:")
    print("Il ranking è basato su una formula che assegna:")
    print("- 80% al market score: stabilità (40%), bassa concentrazione (30%), propensione al pagamento (30%)")
    print("- 20% al technical score: principalmente frequenza di aggiornamento normalizzata")
    print("\nQuesta metodologia potrebbe generare risultati che sembrano in contrasto con alcune")
    print("visualizzazioni grafiche, dove categorie come ART_AND_DESIGN appaiono più favorevoli.")
    print("Il peso significativo dato alla frequenza di aggiornamento spiega perché categorie")
    print("come DATING, con technical score perfetto (1.000), risultino alte in classifica.")

    return results, figures

# Esecuzione dell'analisi
results, figures = perform_category_analysis(apps_clean, reviews_clean)

# Visualizzazione dei grafici
for fig in figures:
    fig.show()

                    concentration  stability  avg_rating
Category                                                
HEALTH_AND_FITNESS          0.313      0.554       4.226
ART_AND_DESIGN              0.265      0.789       4.357
NEWS_AND_MAGAZINES          0.222      0.556       4.143
PARENTING                   0.186      0.749       4.300
WEATHER                     0.180      0.493       4.231

======== TOP 5 CATEGORIE PIÙ PROMETTENTI ========
                final_score  market_score  technical_score
Category                                                  
ENTERTAINMENT         0.785         0.733            0.993
FOOD_AND_DRINK        0.764         0.709            0.984
ART_AND_DESIGN        0.740         0.680            0.978
DATING                0.735         0.669            1.000
SHOPPING              0.733         0.672            0.976

NOTA SULLA METODOLOGIA:
Il ranking è basato su una formula che assegna:
- 80% al market score: stabilità (40%), bassa concentrazione (30%), propensione al pagamento (30%)
- 20% al technical score: principalmente frequenza di aggiornamento normalizzata

Questa metodologia potrebbe generare risultati che sembrano in contrasto con alcune
visualizzazioni grafiche, dove categorie come ART_AND_DESIGN appaiono più favorevoli.
Il peso significativo dato alla frequenza di aggiornamento spiega perché categorie
come DATING, con technical score perfetto (1.000), risultino alte in classifica.

Introduzione¶

1. Import e setup¶

2. Lettura e validazione dei dati¶

3. Pulizia e preparazione dei dati¶

4. Analisi esplorativa dei dati¶

Interpretazione dei risultati¶

Distribuzione delle app per categoria e rating medio¶

Distribuzione dei prezzi delle app a pagamento¶

Mappa competitiva del mercato¶

Analisi delle performance e delle metriche chiave¶

Interpretazione dei risultati¶

Confronto correlazioni (Pearson vs Spearman)¶

Scatter plots delle relazioni principali¶

Evoluzione temporale delle metriche chiave¶

Analisi competitiva e tecnica¶

Interpretazione dei risultati¶

Struttura del mercato per categoria¶

Distribuzione versioni Android per categoria¶

Distribuzione dimensioni app per categoria¶

Conclusione¶

Raccomandazioni strategiche¶

Dashboard Streamlit¶

Category	final_score	market_score	technical_score
ENTERTAINMENT	0.785	0.733	0.993
FOOD_AND_DRINK	0.764	0.709	0.984
ART_AND_DESIGN	0.740	0.680	0.978
DATING	0.735	0.669	1.000
SHOPPING	0.733	0.672	0.976